#include <string>
#include <stdio.h>
#include <vector>
#include <iostream>
#include "document_reader.h"
#include "hse_common.h"
#include "tokenizer.h"
#include "document_meta_datas.h"


namespace hse{

using std::cout;
using std::endl;

int DocumentReader::ID_ = 0;
char *DocumentReader::buffer = NULL;


DocumentReader::DocumentReader(const char* doc_name) 
	: doc_name_(doc_name){
	id_ = ID_++;
}


int DocumentReader::Init() {
	if (!OpenFile()) {
		fprintf(stderr, "Failed to open file %s!", doc_name_.c_str()); 
		return 0;
	}
	ParseWords();
	
	return 1;
}

int DocumentReader::OpenFile() {
	doc_ = fopen(doc_name_.c_str(), "rb");
	if (doc_ == NULL) {
		return 0;
	}
	return 1;
}

void DocumentReader::ParseWords() {
	char word[MAX_TERM_LEN];
	size_t l, total = 0;
	int p = 0;
	int offset = 0;
	while ((l = fread(buffer, sizeof(char), kBufferSize, doc_)) > 0) {
		total += l;
		unsigned int i;
		for (i = 0; i < l; ++i) {
			if (Tokenizer::IsSpliter(buffer[i]) || p >= MAX_TERM_LEN - 1) {
				if (p > 0 ) {
					word[p] = '\0';
					DocWord* docword = new DocWord(word, p, offset + i - p);
					words_.push_back(docword);
					p = 0;
					DEBUG_F(printf("Find word:%s\n", docword->word.c_str()));
				}
			} else {
				word[p++] = tolower(buffer[i]);
			}
		}
		
		offset += l;
	}

	if (p > 0) {
		DEBUG_F(printf("%d", (unsigned char)word[0]));
		word[p] = '\0';
		DocWord* docword = new DocWord(word, p, offset - p);
		words_.push_back(docword);
	}
	length_ = total;
}





DocumentReader::~DocumentReader() {
	for (size_t i = 0; i < words_.size(); ++i) {
		delete words_[i];
	}
	fclose(doc_);
}

}